{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c7903a7a",
   "metadata": {},
   "source": [
    "## datasets"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b7797d06",
   "metadata": {},
   "source": [
    "- `DatasetDict`\n",
    "    - `Dataset`\n",
    "        - 可支持索引"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "fcdd0c34",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-25T14:46:07.379075Z",
     "start_time": "2023-06-25T14:46:02.184987Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6d1e3b31612c4a8ebebc587192b335e1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading builder script: 0.00B [00:00, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using the latest cached version of the module from /home/whaow/.cache/huggingface/modules/datasets_modules/datasets/clinc_oos/abcc41d382f8137f039adc747af44714941e8196e845dfbdd8ae7a7e020e6ba1 (last modified on Wed Jun 14 00:03:25 2023) since it couldn't be found locally at clinc_oos., or remotely on the Hugging Face Hub.\n",
      "Found cached dataset clinc_oos (/home/whaow/.cache/huggingface/datasets/clinc_oos/plus/1.0.0/abcc41d382f8137f039adc747af44714941e8196e845dfbdd8ae7a7e020e6ba1)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "198ac079f0024e17a770451cd09d32a2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "clinc = load_dataset(\"clinc_oos\", \"plus\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c0d5d8ed",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T13:21:22.851267Z",
     "start_time": "2023-06-14T13:21:22.842556Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "datasets.dataset_dict.DatasetDict"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(clinc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "62a1f2d4",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T14:03:20.524292Z",
     "start_time": "2023-06-15T14:03:20.515031Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['text', 'intent'],\n",
       "        num_rows: 15250\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['text', 'intent'],\n",
       "        num_rows: 3100\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['text', 'intent'],\n",
       "        num_rows: 5500\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clinc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f16a8d43",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T13:21:40.579843Z",
     "start_time": "2023-06-14T13:21:40.570477Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['text', 'intent'],\n",
       "    num_rows: 15250\n",
       "})"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clinc['train']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9282722f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T13:22:29.804120Z",
     "start_time": "2023-06-14T13:22:29.793058Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'text': 'what phrase means goodbye in hawaii', 'intent': 61}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clinc['train'][42]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "528ea2b4",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T13:43:27.497743Z",
     "start_time": "2023-06-15T13:43:27.491932Z"
    }
   },
   "outputs": [],
   "source": [
    "# clinc['train'].features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "7e35f18f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T13:25:12.520847Z",
     "start_time": "2023-06-14T13:25:12.511195Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'translate'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clinc['train'].features['intent'].int2str(61)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e980b496",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-29T15:27:16.592637Z",
     "start_time": "2023-06-29T15:27:16.136543Z"
    }
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'clinc' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[1], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m intents \u001b[38;5;241m=\u001b[39m \u001b[43mclinc\u001b[49m[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtrain\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39mfeatures[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mintent\u001b[39m\u001b[38;5;124m'\u001b[39m]\n\u001b[1;32m      2\u001b[0m intents\u001b[38;5;241m.\u001b[39mnum_classes\n",
      "\u001b[0;31mNameError\u001b[0m: name 'clinc' is not defined"
     ]
    }
   ],
   "source": [
    "intents = clinc['train'].features['intent']\n",
    "intents.num_classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "27f952f2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T14:03:12.201106Z",
     "start_time": "2023-06-15T14:03:12.191597Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "translate\n",
      "61\n"
     ]
    }
   ],
   "source": [
    "# 维护类别id及其语义信息\n",
    "# intents = clinc['train'].features['intent']\n",
    "# intents = clinc['test'].features['intent']\n",
    "intents = clinc['validation'].features['intent']\n",
    "print(intents.int2str(61))\n",
    "print(intents.str2int(intents.int2str(61)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eeb80c1e",
   "metadata": {},
   "source": [
    "### map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d3795337",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-25T14:45:42.770353Z",
     "start_time": "2023-06-25T14:45:41.901450Z"
    }
   },
   "outputs": [],
   "source": [
    "from transformers import AutoTokenizer\n",
    "tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "635a37c6",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-25T14:46:15.439936Z",
     "start_time": "2023-06-25T14:46:15.424450Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['text', 'intent'],\n",
       "        num_rows: 15250\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['text', 'intent'],\n",
       "        num_rows: 3100\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['text', 'intent'],\n",
       "        num_rows: 5500\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clinc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f35d7f7b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-25T14:51:32.808812Z",
     "start_time": "2023-06-25T14:51:32.798762Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_items([('train', Dataset({\n",
       "    features: ['text', 'intent'],\n",
       "    num_rows: 15250\n",
       "})), ('validation', Dataset({\n",
       "    features: ['text', 'intent'],\n",
       "    num_rows: 3100\n",
       "})), ('test', Dataset({\n",
       "    features: ['text', 'intent'],\n",
       "    num_rows: 5500\n",
       "}))])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clinc.items()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7d5456b8",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-25T14:51:45.655986Z",
     "start_time": "2023-06-25T14:51:45.646086Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "datasets.arrow_dataset.Dataset"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(clinc['train'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "997477a3",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-25T14:53:22.727623Z",
     "start_time": "2023-06-25T14:53:22.674900Z"
    }
   },
   "outputs": [],
   "source": [
    "clinc.map??"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "9380ddd2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-25T14:54:54.168354Z",
     "start_time": "2023-06-25T14:54:54.103037Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached processed dataset at /home/whaow/.cache/huggingface/datasets/clinc_oos/plus/1.0.0/abcc41d382f8137f039adc747af44714941e8196e845dfbdd8ae7a7e020e6ba1/cache-7f8f509bccf0bd4e.arrow\n",
      "Loading cached processed dataset at /home/whaow/.cache/huggingface/datasets/clinc_oos/plus/1.0.0/abcc41d382f8137f039adc747af44714941e8196e845dfbdd8ae7a7e020e6ba1/cache-cc1b9212398c2167.arrow\n",
      "Loading cached processed dataset at /home/whaow/.cache/huggingface/datasets/clinc_oos/plus/1.0.0/abcc41d382f8137f039adc747af44714941e8196e845dfbdd8ae7a7e020e6ba1/cache-5bcc0db64fa44b4e.arrow\n"
     ]
    }
   ],
   "source": [
    "# batched 默认为 Fasle，batch_size 默认为 1000\n",
    "# 也就是什么都不指定时，默认的处理为一次处理一个样本；\n",
    "# batched == False 时，默认为 batch_size = 1000\n",
    "clinc_enc = clinc.map(lambda batch: tokenizer(batch['text'], truncation=True), \n",
    "                      batched=True, \n",
    "#                       batch_size=1000,\n",
    "                      remove_columns=[\"text\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cec1118c",
   "metadata": {},
   "source": [
    "## metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "4a57a572",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T13:43:45.440773Z",
     "start_time": "2023-06-15T13:43:45.435153Z"
    }
   },
   "outputs": [],
   "source": [
    "from datasets import load_metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "7c50f253",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T13:44:17.097812Z",
     "start_time": "2023-06-15T13:44:17.092054Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['HTTP_PROXY'] = 'http://127.0.0.1:7890'\n",
    "os.environ['HTTPS_PROXY'] = 'http://127.0.0.1:7890'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "d622002f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T13:44:20.863680Z",
     "start_time": "2023-06-15T13:44:18.056274Z"
    }
   },
   "outputs": [],
   "source": [
    "accuracy_score = load_metric('accuracy')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "7f051c2c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T14:06:02.901070Z",
     "start_time": "2023-06-15T14:06:02.880246Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.5}"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# accuracy_score\n",
    "# references: true labels;\n",
    "accuracy_score.compute(predictions=[1, 1, 2, 2], references=[2, 1, 2, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2010b43d",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# from datasets import load_metric\n",
    "# accuracy_score = load_metric('accuracy')\n",
    "import evaluate\n",
    "accuracy_score = evaluate.load('accuracy')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ef44a222",
   "metadata": {},
   "source": [
    "### glue metric"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a12f879b",
   "metadata": {},
   "source": [
    "> General Language Understanding Evaluation (GLUE) benchmark"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "05e9f832",
   "metadata": {},
   "source": [
    "The **M**icrosoft **R**esearch **P**araphrase **C**orpus (Dolan & Brockett, 2005) is a corpus of sentence pairs automatically extracted from online news sources, with human annotations for whether the sentences in the pair are semantically equivalent."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d46cf966",
   "metadata": {},
   "outputs": [],
   "source": [
    "mrpc_metric = load_metric(\"glue\", \"mrpc\")\n",
    "sst2_metric = load_metric('glue', 'sst2') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "2ec5ab06",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-25T15:09:26.997889Z",
     "start_time": "2023-06-25T15:09:23.005996Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "EvaluationModule(name: \"glue\", module_type: \"metric\", features: {'predictions': Value(dtype='int64', id=None), 'references': Value(dtype='int64', id=None)}, usage: \"\"\"\n",
       "Compute GLUE evaluation metric associated to each GLUE dataset.\n",
       "Args:\n",
       "    predictions: list of predictions to score.\n",
       "        Each translation should be tokenized into a list of tokens.\n",
       "    references: list of lists of references for each translation.\n",
       "        Each reference should be tokenized into a list of tokens.\n",
       "Returns: depending on the GLUE subset, one or several of:\n",
       "    \"accuracy\": Accuracy\n",
       "    \"f1\": F1 score\n",
       "    \"pearson\": Pearson Correlation\n",
       "    \"spearmanr\": Spearman Correlation\n",
       "    \"matthews_correlation\": Matthew Correlation\n",
       "Examples:\n",
       "\n",
       "    >>> glue_metric = evaluate.load('glue', 'sst2')  # 'sst2' or any of [\"mnli\", \"mnli_mismatched\", \"mnli_matched\", \"qnli\", \"rte\", \"wnli\", \"hans\"]\n",
       "    >>> references = [0, 1]\n",
       "    >>> predictions = [0, 1]\n",
       "    >>> results = glue_metric.compute(predictions=predictions, references=references)\n",
       "    >>> print(results)\n",
       "    {'accuracy': 1.0}\n",
       "\n",
       "    >>> glue_metric = evaluate.load('glue', 'mrpc')  # 'mrpc' or 'qqp'\n",
       "    >>> references = [0, 1]\n",
       "    >>> predictions = [0, 1]\n",
       "    >>> results = glue_metric.compute(predictions=predictions, references=references)\n",
       "    >>> print(results)\n",
       "    {'accuracy': 1.0, 'f1': 1.0}\n",
       "\n",
       "    >>> glue_metric = evaluate.load('glue', 'stsb')\n",
       "    >>> references = [0., 1., 2., 3., 4., 5.]\n",
       "    >>> predictions = [0., 1., 2., 3., 4., 5.]\n",
       "    >>> results = glue_metric.compute(predictions=predictions, references=references)\n",
       "    >>> print({\"pearson\": round(results[\"pearson\"], 2), \"spearmanr\": round(results[\"spearmanr\"], 2)})\n",
       "    {'pearson': 1.0, 'spearmanr': 1.0}\n",
       "\n",
       "    >>> glue_metric = evaluate.load('glue', 'cola')\n",
       "    >>> references = [0, 1]\n",
       "    >>> predictions = [0, 1]\n",
       "    >>> results = glue_metric.compute(predictions=predictions, references=references)\n",
       "    >>> print(results)\n",
       "    {'matthews_correlation': 1.0}\n",
       "\"\"\", stored examples: 0)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# import evaluate\n",
    "# evaluate.load('glue', 'mrpc')\n",
    "# evaluate.load('glue', 'sst2')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f5bf8f24",
   "metadata": {},
   "source": [
    "### BLUE/ROUGE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "56838497",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-07-14T13:26:20.744147Z",
     "start_time": "2023-07-14T13:26:17.519723Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://pypi.tuna.tsinghua.edu.cn/simple\n",
      "Requirement already satisfied: evaluate in /home/whaow/anaconda3/lib/python3.10/site-packages (0.4.0)\n",
      "Requirement already satisfied: rouge-score in /home/whaow/anaconda3/lib/python3.10/site-packages (0.1.2)\n",
      "Requirement already satisfied: datasets>=2.0.0 in /home/whaow/anaconda3/lib/python3.10/site-packages (from evaluate) (2.10.1)\n",
      "Requirement already satisfied: responses<0.19 in /home/whaow/anaconda3/lib/python3.10/site-packages (from evaluate) (0.18.0)\n",
      "Requirement already satisfied: huggingface-hub>=0.7.0 in /home/whaow/anaconda3/lib/python3.10/site-packages (from evaluate) (0.14.1)\n",
      "Requirement already satisfied: fsspec[http]>=2021.05.0 in /home/whaow/anaconda3/lib/python3.10/site-packages (from evaluate) (2022.11.0)\n",
      "Requirement already satisfied: requests>=2.19.0 in /home/whaow/anaconda3/lib/python3.10/site-packages (from evaluate) (2.30.0)\n",
      "Requirement already satisfied: tqdm>=4.62.1 in /home/whaow/anaconda3/lib/python3.10/site-packages (from evaluate) (4.65.0)\n",
      "Requirement already satisfied: xxhash in /home/whaow/anaconda3/lib/python3.10/site-packages (from evaluate) (3.2.0)\n",
      "Requirement already satisfied: dill in /home/whaow/anaconda3/lib/python3.10/site-packages (from evaluate) (0.3.6)\n",
      "Requirement already satisfied: packaging in /home/whaow/anaconda3/lib/python3.10/site-packages (from evaluate) (23.0)\n",
      "Requirement already satisfied: multiprocess in /home/whaow/anaconda3/lib/python3.10/site-packages (from evaluate) (0.70.14)\n",
      "Requirement already satisfied: pandas in /home/whaow/anaconda3/lib/python3.10/site-packages (from evaluate) (1.5.3)\n",
      "Requirement already satisfied: numpy>=1.17 in /home/whaow/anaconda3/lib/python3.10/site-packages (from evaluate) (1.23.5)\n",
      "Requirement already satisfied: six>=1.14.0 in /home/whaow/anaconda3/lib/python3.10/site-packages (from rouge-score) (1.16.0)\n",
      "Requirement already satisfied: absl-py in /home/whaow/anaconda3/lib/python3.10/site-packages (from rouge-score) (1.4.0)\n",
      "Requirement already satisfied: nltk in /home/whaow/anaconda3/lib/python3.10/site-packages (from rouge-score) (3.7)\n",
      "Requirement already satisfied: aiohttp in /home/whaow/anaconda3/lib/python3.10/site-packages (from datasets>=2.0.0->evaluate) (3.8.4)\n",
      "Requirement already satisfied: pyarrow>=6.0.0 in /home/whaow/anaconda3/lib/python3.10/site-packages (from datasets>=2.0.0->evaluate) (11.0.0)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /home/whaow/anaconda3/lib/python3.10/site-packages (from datasets>=2.0.0->evaluate) (6.0)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/whaow/anaconda3/lib/python3.10/site-packages (from huggingface-hub>=0.7.0->evaluate) (4.5.0)\n",
      "Requirement already satisfied: filelock in /home/whaow/anaconda3/lib/python3.10/site-packages (from huggingface-hub>=0.7.0->evaluate) (3.9.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/whaow/anaconda3/lib/python3.10/site-packages (from requests>=2.19.0->evaluate) (3.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/whaow/anaconda3/lib/python3.10/site-packages (from requests>=2.19.0->evaluate) (2023.5.7)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /home/whaow/anaconda3/lib/python3.10/site-packages (from requests>=2.19.0->evaluate) (2.0.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/whaow/anaconda3/lib/python3.10/site-packages (from requests>=2.19.0->evaluate) (1.26.15)\n",
      "Requirement already satisfied: regex>=2021.8.3 in /home/whaow/anaconda3/lib/python3.10/site-packages (from nltk->rouge-score) (2022.7.9)\n",
      "Requirement already satisfied: joblib in /home/whaow/anaconda3/lib/python3.10/site-packages (from nltk->rouge-score) (1.1.1)\n",
      "Requirement already satisfied: click in /home/whaow/anaconda3/lib/python3.10/site-packages (from nltk->rouge-score) (8.0.4)\n",
      "Requirement already satisfied: python-dateutil>=2.8.1 in /home/whaow/anaconda3/lib/python3.10/site-packages (from pandas->evaluate) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /home/whaow/anaconda3/lib/python3.10/site-packages (from pandas->evaluate) (2023.3)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /home/whaow/anaconda3/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (6.0.4)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /home/whaow/anaconda3/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.8.2)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /home/whaow/anaconda3/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (22.1.0)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /home/whaow/anaconda3/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (4.0.2)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /home/whaow/anaconda3/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.3.3)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /home/whaow/anaconda3/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.3.1)\n"
     ]
    }
   ],
   "source": [
    "!pip install evaluate rouge-score"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "85c329a9",
   "metadata": {},
   "source": [
    "## models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "f2807ddc",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T14:16:36.223436Z",
     "start_time": "2023-06-15T14:16:36.217206Z"
    }
   },
   "outputs": [],
   "source": [
    "# model.state_dict()\n",
    "# model.state_dict().items()\n",
    "\n",
    "# (key, value)\n",
    "# list(model.state_dict().items())[42]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "336px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
